{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 小内存也可以处理大数据\n",
    "# readlines 本身就是迭代器的,这是非常好的."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python 内存分析模块也非常的有意思\n",
    "filepath = './logfile.log'\n",
    "\n",
    "import tracemalloc\n",
    "tracemalloc.start()\n",
    "\n",
    "with open(filepath,'r') as f:\n",
    "    lines = f.readlines()\n",
    "\n",
    "for line in lines:\n",
    "    process_line(line)\n",
    "\n",
    "current, peak = tracemalloc.get_traced_memory()\n",
    "print(f\"Current memory usage: {current / (1024 ** 2)} MB\")\n",
    "print(f\"Peak memory usage: {peak / (1024 ** 2)} MB\")\n",
    "\n",
    "tracemalloc.stop()\n",
    "\n",
    "# 对于一个程序来说,10mb在上没有开始进行工作的情况下就占据了\n",
    "# 这是非常大的一个值了,我们不可能时时刻刻都这么奢侈."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 迭代器协议\n",
    "class LineIterator:\n",
    "    def __init__(self, filepath):\n",
    "        self.file = open(filepath, 'r')\n",
    "\n",
    "    def __iter__(self): \n",
    "        return self\n",
    "\n",
    "    def __next__(self):\n",
    "        line = self.file.readline() \n",
    "        if line:\n",
    "            return line\n",
    "        else:\n",
    "            self.file.close()\n",
    "            raise StopIteration\n",
    "\n",
    "\n",
    "line_iter = LineIterator(filepath)\n",
    "for line in line_iter:\n",
    "    print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加上了内存占用分析\n",
    "tracemalloc.start()\n",
    "\n",
    "class LineIterator:\n",
    "    def __init__(self, filepath):\n",
    "        self.file = open(filepath, 'r')\n",
    "\n",
    "    def __iter__(self): \n",
    "        return self\n",
    "\n",
    "    def __next__(self):\n",
    "        line = self.file.readline() \n",
    "        if line:\n",
    "            return line\n",
    "        else:\n",
    "            self.file.close()\n",
    "            raise StopIteration\n",
    "\n",
    "\n",
    "line_iter = LineIterator(filepath)\n",
    "for line in line_iter:\n",
    "    process_line(line)\n",
    "\n",
    "current, peak = tracemalloc.get_traced_memory()\n",
    "print(f\"Current memory usage: {current / (1024 ** 2) MB\")\n",
    "print(f\"Peak memory usage: {peak / (1024 ** 2) MB\")\n",
    "tracemalloc.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# next魔法方法可以当作一种filter过滤我们需要的file的line\n",
    "tracemalloc.start()\n",
    "\n",
    "class LineIterator:\n",
    "    def __init__(self, filepath):\n",
    "        self.file = open(filepath, 'r')\n",
    "\n",
    "    def __iter__(self): \n",
    "        return self\n",
    "\n",
    "    def __next__(self):\n",
    "        line = self.file.readline() while line:\n",
    "            # 这里的分隔符非常的有用,相当于一个Table的每个列\n",
    "            if line.split('|')[2].strip() == 'Create':\n",
    "                return line\n",
    "            line = self.file.readline()\n",
    "        self.file.close()\n",
    "        raise StopIteration\n",
    "\n",
    "\n",
    "line_iter = LineIterator(filepath)\n",
    "for line in line_iter:\n",
    "    print(line)\n",
    "\n",
    "current, peak = tracemalloc.get_traced_memory()\n",
    "print(f\"Current memory usage: {current / (1024 ** 2) MB\")\n",
    "print(f\"Peak memory usage: {peak / (1024 ** 2) MB\")\n",
    "tracemalloc.stop()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成器类型可以通过yield得到,而且这个生成器直接含有iter和next方法,非常的方便."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 生成器函数\n",
    "def generator(n):\n",
    "    for i in range(n):\n",
    "        print('before yield')\n",
    "        yield i\n",
    "        print('after yield')\n",
    "\n",
    "gen = generator(3)\n",
    "\n",
    "print(next(gen))\n",
    "print('-----')\n",
    "for i in gen:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 重写之前的\n",
    "def line_generator(filepath):\n",
    "    with open(filepath, 'r') as file:\n",
    "        for line in file:\n",
    "            if line.split('|')[2].strip() == 'Create':\n",
    "                yield line\n",
    "            else:\n",
    "                continue\n",
    "\n",
    "line_gen = line_generator(filepath)\n",
    "for line in line_gen:\n",
    "    print(line)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
